Neural Style Transfer on Images

• The notebook is composed of 4 parts.

  • The first two parts: Basic Style Transfer and Output of Different Style Layers demostrate a basic implementation of our algorithm and experiments on 2 content images and 2 style images.
  • The latter parts - Tuning models, Creative Results present details of our experiments on the network and include all the results that we presented in the report.
  • Accordingly, some of the parts have several image outputs and the notebook is long. If you only want to see the implementation of the Neural Style Transfer Algorithm, Basic Style Transfer and Different Style Layer have shown essential work and you can ignore the parts after Tuning models(including Tuning models).
• Some cells are pasted diretly from group member's colab notebooks, hence those cells might look unrun, but essential outputs have been included and demostrated.

Basic Style Transfer

Code Framework

import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
import matplotlib as mpl
import IPython.display as display

import PIL.Image
import functools
import time
import os
from numpy.random import seed
seed(111)

mpl.rcParams['figure.figsize'] = (15,15)
mpl.rcParams['axes.grid'] = False

def to_image(tensor):
  tensor = tensor*255
  tensor = np.array(tensor, dtype=np.uint8)
  if np.ndim(tensor)>3:
    assert tensor.shape[0] == 1
    tensor = tensor[0]
  return PIL.Image.fromarray(tensor)

content_path = 'columbia.jpg'
style_path = 'van_Gogh.jpg'
content_path2 = 'Manhattan.jpg'
style_path2 = 'Pablo_Picasso.jpg'

def load_img(path):

  img = tf.io.read_file(path)
  img = tf.image.decode_image(img, channels=3)
  img = tf.image.convert_image_dtype(img, tf.float32)

  shape = tf.cast(tf.shape(img)[:-1], tf.float32)
  l = max(shape)
  scale = 512 / l

  new_shape = tf.cast(shape * scale, tf.int32)

  img = tf.image.resize(img, new_shape)
  img = img[tf.newaxis, :]
  return img

content_image = load_img(content_path)
style_image = load_img(style_path)

x = tf.keras.applications.vgg19.preprocess_input(content_image*255)
x = tf.image.resize(x, (224, 224))
vgg = tf.keras.applications.VGG19(include_top=True, weights='imagenet')

vgg = tf.keras.applications.VGG19(include_top=False, weights='imagenet')

print()
for layer in vgg.layers:
  print(layer.name)

input_2
block1_conv1
block1_conv2
block1_pool
block2_conv1
block2_conv2
block2_pool
block3_conv1
block3_conv2
block3_conv3
block3_conv4
block3_pool
block4_conv1
block4_conv2
block4_conv3
block4_conv4
block4_pool
block5_conv1
block5_conv2
block5_conv3
block5_conv4
block5_pool

# the layers to extract the content
content_layers = ['block4_conv2'] 

# the layers to extract the style
style_layers = ['block1_conv1',
                'block2_conv1',
                'block3_conv1', 
                'block4_conv1', 
                'block5_conv1']

num_content_layers = len(content_layers)
num_style_layers = len(style_layers)

def vgg_layers(layers):
  # load the pre-trained vgg 
  vgg = tf.keras.applications.VGG19(include_top=False, weights='imagenet')
  vgg.trainable = False

  output = [vgg.get_layer(name).output for name in layers]

  model = tf.keras.Model([vgg.input], output)
  return model

style_extractor = vgg_layers(style_layers)
style_outputs = style_extractor(style_image*255)

def gram_matrix(x):
  shape = tf.shape(x)
  out1 = tf.linalg.einsum('bijc,bijd->bcd', x, x)
  z = tf.cast(shape[1]*shape[2], tf.float32)
  return out1/z

class NeuralTransfer(tf.keras.models.Model):
  def __init__(self, style_layers, content_layers):
    super(NeuralTransfer, self).__init__()
    self.vgg =  vgg_layers(style_layers + content_layers)
    self.vgg.trainable = False
    self.content_layers = content_layers
    self.style_layers = style_layers
    self.num_style_layers = len(style_layers)

  def call(self, img):
    "Expects float input in [0,1]"
    img = img*255.0
    img_pre = tf.keras.applications.vgg19.preprocess_input(img)
    img1 = self.vgg(img_pre)

    style = img1[:self.num_style_layers]
    style = [gram_matrix(s) for s in style]

    style_dict = {style_name:value
                  for style_name, value
                  in zip(self.style_layers, style)}

    content = img1[self.num_style_layers:]

    content_dict = {content_name:value 
                    for content_name, value 
                    in zip(self.content_layers, content)}


    return {'style':style_dict,'content':content_dict}

extractor = NeuralTransfer(style_layers, content_layers)

results = extractor(tf.constant(content_image))

style_results = results['style']

content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']

image = tf.Variable(content_image)

def image_clip(image):
  return tf.clip_by_value(image, clip_value_min=0.0, clip_value_max=1.0)

opt = tf.optimizers.Adam(learning_rate=0.03, beta_1=0.99, epsilon=1e-1)

style_weight=1
content_weight=1e-3

def compute_loss(x):
    style = x['style']
    style_loss = tf.add_n([tf.reduce_mean((style[i]-style_y[i])**2) 
                           for i in style.keys()])
    style_loss *= style_weight/num_style_layers
    
    content = x['content']
    content_loss = tf.add_n([tf.reduce_mean((content[i]-content_y[i])**2) 
                             for i in content.keys()])
    content_loss *= content_weight / num_content_layers

    loss = style_loss + content_loss
    return loss

@tf.function()
def train_iter(image):
  with tf.GradientTape() as tape:
    x = extractor(image)
    loss = compute_loss(x)

  grad = tape.gradient(loss, image)
  opt.apply_gradients([(grad, image)])
  image.assign(image_clip(image))

train_iter(image)
train_iter(image)
train_iter(image)
to_image(image)

# train for 1000 interations
import time
import os
start = time.time()

epochs = 10
iter_per_epoch = 100

step = 0
for n in range(epochs):
  for m in range(iter_per_epoch):
    step += 1
    train_iter(image)

save_path = os.path.join("img_%s.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("Train step: {}".format(step))

end = time.time()
print("Total time: {:.1f}".format(end-start))

'img_1000.png'

Train step: 1000
Total time: 68.3

Experiments

Content 1 x Style 1

tf.random.set_seed(111)

content_image = load_img(content_path)
style_image = load_img(style_path)

plt.subplot(1, 2, 1)
show_image(content_image, 'Content Image')

plt.subplot(1, 2, 2)
show_image(style_image, 'Style Image')

style_extractor = vgg_layers(style_layers)
style_outputs = style_extractor(style_image*255)
results = extractor(tf.constant(content_image))
style_results = results['style']
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']

image_prepared = tf.Variable(content_image)

opt = tf.optimizers.Adam(learning_rate=0.03, beta_1=0.99, epsilon=1e-1)
@tf.function()
def train_iter(image):
  with tf.GradientTape() as tape:
    x = extractor(image)
    loss = compute_loss(x)

  style_weight=1
  content_weight=1e-3  
  grad = tape.gradient(loss, image)
  
  opt.apply_gradients([(grad, image)])
  image.assign(image_clip(image))

start = time.time()
epochs = 1000
image_prepared = basic_style_transfer_prepare(content_path, style_path)
save_path = 'img_c1_s1.jpg'
for n in range(epochs):
  train_iter(image_prepared)

display.display(to_image(image_prepared),save_path)
to_image(image_prepared).save(save_path)

end = time.time()
print("Total time: {:.1f}".format(end-start)) 

'img_c1_s1.jpg'

Total time: 64.3

Content 1 x Style 2

content_image = load_img(content_path)
style_image = load_img(style_path2)

plt.subplot(1, 2, 1)
show_image(content_image, 'Content Image')

plt.subplot(1, 2, 2)
show_image(style_image, 'Style Image')

style_extractor = vgg_layers(style_layers)
style_outputs = style_extractor(style_image*255)
results = extractor(tf.constant(content_image))
style_results = results['style']
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']

image_prepared = tf.Variable(content_image)

opt = tf.optimizers.Adam(learning_rate=0.03, beta_1=0.99, epsilon=1e-1)
@tf.function()
def train_iter(image):
  with tf.GradientTape() as tape:
    x = extractor(image)
    loss = compute_loss(x)

  style_weight=1
  content_weight=1e-3  
  grad = tape.gradient(loss, image)
  
  opt.apply_gradients([(grad, image)])
  image.assign(image_clip(image))

start = time.time()
epochs = 1000
#image_prepared = basic_style_transfer_prepare(content_path, style_path)
save_path = 'img_c1_s2.jpg'
for n in range(epochs):
  train_iter(image_prepared)

display.display(to_image(image_prepared),save_path)
to_image(image_prepared).save(save_path)

end = time.time()
print("Total time: {:.1f}".format(end-start)) 

'img_c1_s2.jpg'

Total time: 63.2

Content 2 x Style 1

content_image = load_img(content_path2)
style_image = load_img(style_path)

plt.subplot(1, 2, 1)
show_image(content_image, 'Content Image')

plt.subplot(1, 2, 2)
show_image(style_image, 'Style Image')

style_extractor = vgg_layers(style_layers)
style_outputs = style_extractor(style_image*255)
results = extractor(tf.constant(content_image))
style_results = results['style']
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']

image_prepared = tf.Variable(content_image)

opt = tf.optimizers.Adam(learning_rate=0.03, beta_1=0.99, epsilon=1e-1)
@tf.function()
def train_iter(image):
  with tf.GradientTape() as tape:
    x = extractor(image)
    loss = compute_loss(x)

  style_weight=1
  content_weight=1e-3  
  grad = tape.gradient(loss, image)
  
  opt.apply_gradients([(grad, image)])
  image.assign(image_clip(image))

start = time.time()
epochs = 1000
#image_prepared = basic_style_transfer_prepare(content_path, style_path)
save_path = 'img_c2_s1.jpg'
for n in range(epochs):
  train_iter(image_prepared)

display.display(to_image(image_prepared),save_path)
to_image(image_prepared).save(save_path)

end = time.time()
print("Total time: {:.1f}".format(end-start)) 

'img_c2_s1.jpg'

Total time: 66.7

Content 2 x Style 2

content_image = load_img(content_path2)
style_image = load_img(style_path2)

plt.subplot(1, 2, 1)
show_image(content_image, 'Content Image')

plt.subplot(1, 2, 2)
show_image(style_image, 'Style Image')

style_extractor = vgg_layers(style_layers)
style_outputs = style_extractor(style_image*255)
results = extractor(tf.constant(content_image))
style_results = results['style']
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']

image_prepared = tf.Variable(content_image)

opt = tf.optimizers.Adam(learning_rate=0.03, beta_1=0.99, epsilon=1e-1)
@tf.function()
def train_iter(image):
  with tf.GradientTape() as tape:
    x = extractor(image)
    loss = compute_loss(x)

  style_weight=1
  content_weight=1e-3  
  grad = tape.gradient(loss, image)
  
  opt.apply_gradients([(grad, image)])
  image.assign(image_clip(image))

start = time.time()
epochs = 1000
save_path = 'img_c2_s2.jpg'
for n in range(epochs):
  train_iter(image_prepared)

display.display(to_image(image_prepared),save_path)
to_image(image_prepared).save(save_path)

end = time.time()
print("Total time: {:.1f}".format(end-start)) 

'img_c2_s2.jpg'

Total time: 67.3

Output of Different Style Layers

basic_image = load_img("white.jpg")

conv1_1

# block1_conv1 : style
style_layers = ['block1_conv1']
extractor = NeuralTransfer(style_layers, content_layers)

results = extractor(tf.constant(basic_image))

style_results = results['style']
num_content_layers = len(content_layers)
num_style_layers = len(style_layers)

content_y = extractor(basic_image)['content']
style_y = extractor(style_image)['style']

image = tf.Variable(basic_image)

def train_iter(image):
  with tf.GradientTape() as tape:
    x = extractor(image)
    loss = compute_loss(x)

  grad = tape.gradient(loss, image)
  opt.apply_gradients([(grad, image)])
  image.assign(image_clip(image))

def compute_loss(x):
    style = x['style']
    style_loss = tf.add_n([tf.reduce_mean((style[i]-style_y[i])**2) 
                           for i in style.keys()])
    style_loss *= style_weight/num_style_layers
    
    content = x['content']
    content_loss = tf.add_n([tf.reduce_mean((content[i]-content_y[i])**2) 
                             for i in content.keys()])
    content_loss *= content_weight / num_content_layers

    loss = content_loss + style_loss
    return loss


step = 0
start = time.time()
for i in range(1000):
  step += 1
  train_iter(image)
  
save_path = os.path.join("style_%s_1.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))

end = time.time()
print("Total time: {:.1f}".format(end-start))

'style_1000_1.png'

train step: 1000
Total time: 109.9

# block1_conv1 : style -> content
style_layers = ['block1_conv1']
extractor = NeuralTransfer(style_layers, content_layers)

results = extractor(tf.constant(content_image))

style_results = results['style']
num_content_layers = len(content_layers)
num_style_layers = len(style_layers)

content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']

def compute_loss(x):
    style = x['style']
    style_loss = tf.add_n([tf.reduce_mean((style[i]-style_y[i])**2) 
                           for i in style.keys()])
    style_loss *= style_weight/num_style_layers
    
    content = x['content']
    content_loss = tf.add_n([tf.reduce_mean((content[i]-content_y[i])**2) 
                             for i in content.keys()])
    content_loss *= content_weight / num_content_layers

    loss = content_loss + style_loss
    return loss



image = tf.Variable(content_image)

step = 0
start = time.time()
for i in range(1000):
  step += 1
  train_iter(image)

save_path = os.path.join("content_%s_1.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))

end = time.time()
print("Total time: {:.1f}".format(end-start))

'content_1000_1.png'

train step: 1000
Total time: 93.9

conv1_1, conv2_1

# ['block1_conv1','block2_conv1']:style
style_layers = ['block1_conv1',
                'block2_conv1']

num_content_layers = len(content_layers)
num_style_layers = len(style_layers)

extractor = NeuralTransfer(style_layers, content_layers)

results = extractor(tf.constant(basic_image))

style_results = results['style']
num_content_layers = len(content_layers)
num_style_layers = len(style_layers)

content_y = extractor(basic_image)['content']
style_y = extractor(style_image)['style']

def compute_loss(x):
    style = x['style']
    style_loss = tf.add_n([tf.reduce_mean((style[i]-style_y[i])**2) 
                           for i in style.keys()])
    style_loss *= style_weight/num_style_layers
    
    content = x['content']
    content_loss = tf.add_n([tf.reduce_mean((content[i]-content_y[i])**2) 
                             for i in content.keys()])
    content_loss *= content_weight / num_content_layers

    loss = content_loss + style_loss
    return loss

image = tf.Variable(basic_image)

step = 0
start = time.time()
for i in range(1000):
  step += 1
  train_iter(image)

save_path = os.path.join("style_%s_2.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))

end = time.time()
print("Total time: {:.1f}".format(end-start))

'style_1000_2.png'

train step: 1000
Total time: 112.9

# style -> content
style_layers = ['block1_conv1',
                'block2_conv1']
extractor = NeuralTransfer(style_layers, content_layers)

results = extractor(tf.constant(content_image))

style_results = results['style']
num_content_layers = len(content_layers)
num_style_layers = len(style_layers)

content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']

def compute_loss(x):
    style = x['style']
    style_loss = tf.add_n([tf.reduce_mean((style[i]-style_y[i])**2) 
                           for i in style.keys()])
    style_loss *= style_weight/num_style_layers
    
    content = x['content']
    content_loss = tf.add_n([tf.reduce_mean((content[i]-content_y[i])**2) 
                             for i in content.keys()])
    content_loss *= content_weight / num_content_layers

    loss = content_loss + style_loss
    return loss



image = tf.Variable(content_image)

step = 0
start = time.time()
for i in range(1000):
  step += 1
  train_iter(image)

save_path = os.path.join("content_%s_2.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))

end = time.time()
print("Total time: {:.1f}".format(end-start))

'content_1000_2.png'

train step: 1000
Total time: 96.5

conv1_1, conv2_1, conv3_1

# style 
style_layers = ['block1_conv1',
                'block2_conv1',
                'block3_conv1']

num_content_layers = len(content_layers)
num_style_layers = len(style_layers)

extractor = NeuralTransfer(style_layers, content_layers)

results = extractor(tf.constant(basic_image))

style_results = results['style']
num_content_layers = len(content_layers)
num_style_layers = len(style_layers)

content_y = extractor(basic_image)['content']
style_y = extractor(style_image)['style']

def compute_loss(x):
    style = x['style']
    style_loss = tf.add_n([tf.reduce_mean((style[i]-style_y[i])**2) 
                           for i in style.keys()])
    style_loss *= style_weight/num_style_layers
    
    content = x['content']
    content_loss = tf.add_n([tf.reduce_mean((content[i]-content_y[i])**2) 
                             for i in content.keys()])
    content_loss *= content_weight / num_content_layers

    loss = content_loss + style_loss
    return loss

image = tf.Variable(basic_image)

step = 0
start = time.time()
for i in range(1000):
  step += 1
  train_iter(image)

save_path = os.path.join("style_%s_3.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))

end = time.time()
print("Total time: {:.1f}".format(end-start))

'style_1000_3.png'

train step: 1000
Total time: 115.6

# style -> content
style_layers = ['block1_conv1',
                'block2_conv1',
                'block3_conv1']
extractor = NeuralTransfer(style_layers, content_layers)

results = extractor(tf.constant(content_image))

style_results = results['style']
num_content_layers = len(content_layers)
num_style_layers = len(style_layers)

content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']

def compute_loss(x):
    style = x['style']
    style_loss = tf.add_n([tf.reduce_mean((style[i]-style_y[i])**2) 
                           for i in style.keys()])
    style_loss *= style_weight/num_style_layers
    
    content = x['content']
    content_loss = tf.add_n([tf.reduce_mean((content[i]-content_y[i])**2) 
                             for i in content.keys()])
    content_loss *= content_weight / num_content_layers

    loss = content_loss + style_loss
    return loss



image = tf.Variable(content_image)

step = 0
start = time.time()
for i in range(1000):
  step += 1
  train_iter(image)

save_path = os.path.join("content_%s_3.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))

end = time.time()
print("Total time: {:.1f}".format(end-start))

'content_1000_3.png'

train step: 1000
Total time: 99.4

conv1_1, conv2_1, conv3_1, conv4_1

# block1_conv1, block2_conv1, block3_conv1, block4_conv1
# style
style_layers = ['block1_conv1',
                'block2_conv1',
                'block3_conv1', 
                'block4_conv1']

num_content_layers = len(content_layers)
num_style_layers = len(style_layers)

extractor = NeuralTransfer(style_layers, content_layers)

results = extractor(tf.constant(basic_image))

style_results = results['style']
num_content_layers = len(content_layers)
num_style_layers = len(style_layers)

content_y = extractor(basic_image)['content']
style_y = extractor(style_image)['style']

def compute_loss(x):
    style = x['style']
    style_loss = tf.add_n([tf.reduce_mean((style[i]-style_y[i])**2) 
                           for i in style.keys()])
    style_loss *= style_weight/num_style_layers
    
    content = x['content']
    content_loss = tf.add_n([tf.reduce_mean((content[i]-content_y[i])**2) 
                             for i in content.keys()])
    content_loss *= content_weight / num_content_layers

    loss = content_loss + style_loss
    return loss

image = tf.Variable(basic_image)

step = 0
start = time.time()
for i in range(1000):
  step += 1
  train_iter(image)

save_path = os.path.join("style_%s_4.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))

end = time.time()
print("Total time: {:.1f}".format(end-start))

'style_1000_4.png'

train step: 1000
Total time: 118.4

# style -> content
style_layers = ['block1_conv1',
                'block2_conv1',
                'block3_conv1', 
                'block4_conv1']
extractor = NeuralTransfer(style_layers, content_layers)

results = extractor(tf.constant(content_image))

style_results = results['style']
num_content_layers = len(content_layers)
num_style_layers = len(style_layers)

content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']

def compute_loss(x):
    style = x['style']
    style_loss = tf.add_n([tf.reduce_mean((style[i]-style_y[i])**2) 
                           for i in style.keys()])
    style_loss *= style_weight/num_style_layers
    
    content = x['content']
    content_loss = tf.add_n([tf.reduce_mean((content[i]-content_y[i])**2) 
                             for i in content.keys()])
    content_loss *= content_weight / num_content_layers

    loss = content_loss + style_loss
    return loss



image = tf.Variable(content_image)

step = 0
start = time.time()
for i in range(1000):
  step += 1
  train_iter(image)

save_path = os.path.join("content_%s_4.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))

end = time.time()
print("Total time: {:.1f}".format(end-start))

'content_1000_4.png'

train step: 1000
Total time: 101.4

conv1_1, conv2_1, conv3_1, conv4_1, conv5_1

# style
style_layers = ['block1_conv1',
                'block2_conv1',
                'block3_conv1', 
                'block4_conv1', 
                'block5_conv1']

num_content_layers = len(content_layers)
num_style_layers = len(style_layers)

extractor = NeuralTransfer(style_layers, content_layers)

results = extractor(tf.constant(basic_image))

style_results = results['style']
num_content_layers = len(content_layers)
num_style_layers = len(style_layers)

content_y = extractor(basic_image)['content']
style_y = extractor(style_image)['style']

image = tf.Variable(basic_image)


step = 0
start = time.time()
for i in range(1000):
  step += 1
  train_iter(image)

save_path = os.path.join("style_%s_5.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))

end = time.time()
print("Total time: {:.1f}".format(end-start))

'style_1000_5.png'

train step: 1000
Total time: 141.5

# style -> content
style_layers = ['block1_conv1',
                'block2_conv1',
                'block3_conv1', 
                'block4_conv1', 
                'block5_conv1']

num_content_layers = len(content_layers)
num_style_layers = len(style_layers)

extractor = NeuralTransfer(style_layers, content_layers)

results = extractor(tf.constant(content_image))

style_results = results['style']
num_content_layers = len(content_layers)
num_style_layers = len(style_layers)

content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']

def compute_loss(x):
    style = x['style']
    style_loss = tf.add_n([tf.reduce_mean((style[i]-style_y[i])**2) 
                           for i in style.keys()])
    style_loss *= style_weight/num_style_layers
    
    content = x['content']
    content_loss = tf.add_n([tf.reduce_mean((content[i]-content_y[i])**2) 
                             for i in content.keys()])
    content_loss *= content_weight / num_content_layers

    loss = content_loss + style_loss
    return loss



image = tf.Variable(content_image)

step = 0
start = time.time()
for i in range(1000):
  step += 1
  train_iter(image)

save_path = os.path.join("content_%s_5.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))

end = time.time()
print("Total time: {:.1f}".format(end-start))

'content_1000_5.png'

train step: 1000
Total time: 122.1

Tuning models

Number of Iterations

Iterations = [10, 50, 100, 500, 1000, 5000, 10000,12000,14000,15000]

style_layers = ['block1_conv1',
                'block2_conv1',
                'block3_conv1', 
                'block4_conv1', 
                'block5_conv1']

num_content_layers = len(content_layers)
num_style_layers = len(style_layers)

extractor = NeuralTransfer(style_layers, content_layers)

results = extractor(tf.constant(content_image))

style_results = results['style']
num_content_layers = len(content_layers)
num_style_layers = len(style_layers)

content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']

image = tf.Variable(content_image)

start = time.time()


step = 0
for i in range(15000):
  step += 1
  train_iter(image)
  if step in [10, 50, 100, 500, 1000, 5000, 10000,12000,14000,15000]:
    save_path = os.path.join("img_%s.png"%(step))
    display.display(to_image(image),save_path)
    to_image(image).save(save_path)
    print("train step: {}".format(step))
    end = time.time()
    print("current time: {:.1f}".format(end-start))

end = time.time()
print("Total time: {:.1f}".format(end-start))

'img_10.png'

train step: 10
current time: 1.3

'img_50.png'

train step: 50
current time: 6.3

'img_100.png'

train step: 100
current time: 12.5

'img_500.png'

train step: 500
current time: 61.7

'img_1000.png'

train step: 1000
current time: 122.5

'img_5000.png'

train step: 5000
current time: 608.4

'img_10000.png'

train step: 10000
current time: 1215.8

'img_12000.png'

train step: 12000
current time: 1458.8

'img_14000.png'

train step: 14000
current time: 1701.8

'img_15000.png'

train step: 15000
current time: 1823.3
Total time: 1823.3

Learning Rate

Iteration: 1000

# lr = 0.01
content_image = load_img(content_path)
style_image = load_img(style_path)
random_seed = 111
tf.random.set_seed(random_seed)

style_extractor = vgg_layers(style_layers)
style_outputs = style_extractor(style_image*255)
results = extractor(tf.constant(content_image))
style_results = results['style']
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']

image_prepared = tf.Variable(content_image)
opt = tf.optimizers.Adam(learning_rate=0.01, beta_1=0.99, epsilon=1e-1)

@tf.function()
def train_iter(image):
  with tf.GradientTape() as tape:
    x = extractor(image)
    loss = compute_loss(x)

  style_weight=1
  content_weight=1e-3  
  grad = tape.gradient(loss, image)
  
  opt.apply_gradients([(grad, image)])
  image.assign(image_clip(image))


step = 0
image = tf.Variable(content_image)
start = time.time()
for i in range(1000):
  step += 1
  train_iter(image)

save_path = os.path.join("img_%s_0.01.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))

end = time.time()
print("Total time: {:.1f}".format(end-start))

'img_1000_0.01.png'

train step: 1000
Total time: 69.1

# lr = 0.05
content_image = load_img(content_path)
style_image = load_img(style_path)
random_seed = 111

style_extractor = vgg_layers(style_layers)
style_outputs = style_extractor(style_image*255)
results = extractor(tf.constant(content_image))
style_results = results['style']
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']
tf.random.set_seed(random_seed)
opt = tf.optimizers.Adam(learning_rate=0.05, beta_1=0.99, epsilon=1e-1)

image_prepared = tf.Variable(content_image)

@tf.function()
def train_iter(image):
  with tf.GradientTape() as tape:
    x = extractor(image)
    loss = compute_loss(x)

  style_weight=1
  content_weight=1e-3  
  grad = tape.gradient(loss, image)
  
  opt.apply_gradients([(grad, image)])
  image.assign(image_clip(image))


step = 0
image = tf.Variable(content_image)
start = time.time()
for i in range(1000):
  step += 1
  train_iter(image)

save_path = os.path.join("img_%s_0.05.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))

end = time.time()
print("Total time: {:.1f}".format(end-start))

'img_1000_0.05.png'

train step: 1000
Total time: 68.6

# lr = 0.1
content_image = load_img(content_path)
style_image = load_img(style_path)
random_seed = 111

style_extractor = vgg_layers(style_layers)
style_outputs = style_extractor(style_image*255)
results = extractor(tf.constant(content_image))
style_results = results['style']
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']
tf.random.set_seed(random_seed)
opt = tf.optimizers.Adam(learning_rate=0.1, beta_1=0.99, epsilon=1e-1)

image_prepared = tf.Variable(content_image)

@tf.function()
def train_iter(image):
  with tf.GradientTape() as tape:
    x = extractor(image)
    loss = compute_loss(x)

  style_weight=1
  content_weight=1e-3  
  grad = tape.gradient(loss, image)
  
  opt.apply_gradients([(grad, image)])
  image.assign(image_clip(image))


step = 0
image = tf.Variable(content_image)
start = time.time()
for i in range(1000):
  step += 1
  train_iter(image)

save_path = os.path.join("img_%s_0.1.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))

end = time.time()
print("Total time: {:.1f}".format(end-start))

'img_1000_0.1.png'

train step: 1000
Total time: 68.7

Iteration: 2000

# lr = 0.01
content_image = load_img(content_path)
style_image = load_img(style_path)
random_seed = 111

style_extractor = vgg_layers(style_layers)
style_outputs = style_extractor(style_image*255)
results = extractor(tf.constant(content_image))
style_results = results['style']
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']
tf.random.set_seed(random_seed)
opt = tf.optimizers.Adam(learning_rate=0.01, beta_1=0.99, epsilon=1e-1)

image_prepared = tf.Variable(content_image)

@tf.function()
def train_iter(image):
  with tf.GradientTape() as tape:
    x = extractor(image)
    loss = compute_loss(x)

  style_weight=1
  content_weight=1e-3  
  grad = tape.gradient(loss, image)
  
  opt.apply_gradients([(grad, image)])
  image.assign(image_clip(image))


step = 0
image = tf.Variable(content_image)
start = time.time()
for i in range(2000):
  step += 1
  train_iter(image)

save_path = os.path.join("img_%s_0.01.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))

end = time.time()
print("Total time: {:.1f}".format(end-start))

'img_2000_0.01.png'

train step: 2000
Total time: 136.9

# lr = 0.05
content_image = load_img(content_path)
style_image = load_img(style_path)
random_seed = 111

style_extractor = vgg_layers(style_layers)
style_outputs = style_extractor(style_image*255)
results = extractor(tf.constant(content_image))
style_results = results['style']
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']
tf.random.set_seed(random_seed)
opt = tf.optimizers.Adam(learning_rate=0.05, beta_1=0.99, epsilon=1e-1)

image_prepared = tf.Variable(content_image)

@tf.function()
def train_iter(image):
  with tf.GradientTape() as tape:
    x = extractor(image)
    loss = compute_loss(x)

  style_weight=1
  content_weight=1e-3  
  grad = tape.gradient(loss, image)
  
  opt.apply_gradients([(grad, image)])
  image.assign(image_clip(image))


step = 0
image = tf.Variable(content_image)
start = time.time()
for i in range(2000):
  step += 1
  train_iter(image)

save_path = os.path.join("img_%s_0.05.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))

end = time.time()
print("Total time: {:.1f}".format(end-start))

'img_2000_0.05.png'

train step: 2000
Total time: 136.9

# lr = 0.10
content_image = load_img(content_path)
style_image = load_img(style_path)
random_seed = 111

style_extractor = vgg_layers(style_layers)
style_outputs = style_extractor(style_image*255)
results = extractor(tf.constant(content_image))
style_results = results['style']
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']
tf.random.set_seed(random_seed)
opt = tf.optimizers.Adam(learning_rate=0.1, beta_1=0.99, epsilon=1e-1)

image_prepared = tf.Variable(content_image)

@tf.function()
def train_iter(image):
  with tf.GradientTape() as tape:
    x = extractor(image)
    loss = compute_loss(x)

  style_weight=1
  content_weight=1e-3  
  grad = tape.gradient(loss, image)
  
  opt.apply_gradients([(grad, image)])
  image.assign(image_clip(image))


step = 0
image = tf.Variable(content_image)
start = time.time()
for i in range(2000):
  step += 1
  train_iter(image)

save_path = os.path.join("img_%s_0.1.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))

end = time.time()
print("Total time: {:.1f}".format(end-start))

'img_2000_0.1.png'

train step: 2000
Total time: 137.2

Layers

Change content layer from conv4-2 to conv5-2

# the layers to extract the content
content_layers = ['block5_conv2'] 

# the layers to extract the style
style_layers = ['block1_conv1',
                'block2_conv1',
                'block3_conv1', 
                'block4_conv1', 
                'block5_conv1']

num_content_layers = len(content_layers)
num_style_layers = len(style_layers)

extractor = NeuralTransfer(style_layers, content_layers)

content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']

image = tf.Variable(content_image)

@tf.function()
def train_iter(image):
  with tf.GradientTape() as tape:
    x = extractor(image)
    loss = compute_loss(x)

  grad = tape.gradient(loss, image)
  opt.apply_gradients([(grad, image)])
  image.assign(image_clip(image))

opt = tf.optimizers.Adam(learning_rate=0.05, beta_1=0.99, epsilon=1e-1)


start = time.time()

epochs = 5
iter_per_epoch = 200

step = 0
for n in range(epochs):
  for m in range(iter_per_epoch):
    step += 1
    train_iter(image)
    print(".", end='')
  display.clear_output(wait=True)
  display.display(to_image(image),save_path)
  print("Train step: {}".format(step))

save_path = os.path.join("img_%s_cont5-2.png"%(step))
to_image(image).save(save_path)
end = time.time()
print("Total time: {:.1f}".format(end-start))

'img_1000.png'

Train step: 1000
Total time: 72.5

Change style layer set (change the first one from conv1-1 to conv1-2)

# the layers to extract the content
content_layers = ['block4_conv2'] 

# the layers to extract the style
style_layers = ['block1_conv2',
                'block2_conv1',
                'block3_conv1', 
                'block4_conv1', 
                'block5_conv1']

num_content_layers = len(content_layers)
num_style_layers = len(style_layers)

extractor = NeuralTransfer(style_layers, content_layers)

content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']

image = tf.Variable(content_image)

@tf.function()
def train_iter(image):
  with tf.GradientTape() as tape:
    x = extractor(image)
    loss = compute_loss(x)

  grad = tape.gradient(loss, image)
  opt.apply_gradients([(grad, image)])
  image.assign(image_clip(image))

opt = tf.optimizers.Adam(learning_rate=0.03, beta_1=0.99, epsilon=1e-1)


start = time.time()

epochs = 5
iter_per_epoch = 200

step = 0
for n in range(epochs):
  for m in range(iter_per_epoch):
    step += 1
    train_iter(image)
    print(".", end='')
  display.clear_output(wait=True)
  display.display(to_image(image),save_path)
  print("Train step: {}".format(step))

save_path = os.path.join("img_%s_sty1-2.png"%(step))
to_image(image).save(save_path)

end = time.time()
print("Total time: {:.1f}".format(end-start))

'img_1000_cont5-2.png'

Train step: 1000
Total time: 71.9

Content-Style Weights/Ratio

def compute_loss_with_ratio(x, content_weight, style_weight):
    style = x['style']
    style_loss = tf.add_n([tf.reduce_mean((style[i]-style_y[i])**2) 
                           for i in style.keys()])
    style_loss *= style_weight/num_style_layers
    
    content = x['content']
    content_loss = tf.add_n([tf.reduce_mean((content[i]-content_y[i])**2) 
                             for i in content.keys()])
    content_loss *= content_weight / num_content_layers

    loss = style_loss + content_loss
    return loss

@tf.function()
def train_iter_with_ratio(image, content_weight, style_weight):
  with tf.GradientTape() as tape:
    x = extractor(image)
    loss = compute_loss_with_ratio(x, content_weight, style_weight)

  grad = tape.gradient(loss, image)
  opt.apply_gradients([(grad, image)])
  image.assign(image_clip(image))

# alpha: content, beta: style
# alpha/beta: 1e-4
content_image = load_img(content_path)
image_prepared = tf.Variable(content_image)

content_weight=1e-4
style_weight=1

@tf.function()
def train_iter_with_ratio(image, content_weight, style_weight):
  with tf.GradientTape() as tape:
    x = extractor(image)
    loss = compute_loss_with_ratio(x, content_weight, style_weight)

  grad = tape.gradient(loss, image)
  opt.apply_gradients([(grad, image)])
  image.assign(image_clip(image))

start = time.time()
epochs = 1000
save_path = 'img_weight_1e-4.jpg'
for n in range(epochs):
  train_iter_with_ratio(image_prepared, content_weight, style_weight)

display.display(to_image(image_prepared),save_path)
to_image(image_prepared).save(save_path)

end = time.time()
print("Total time: {:.1f}".format(end-start)) 

'img_weight_1e-4.jpg'

Total time: 74.8

# alpha: content, beta: style
# alpha/beta: 1e-2
content_image = load_img(content_path)
image_prepared = tf.Variable(content_image)

content_weight=1e-2
style_weight=1

@tf.function()
def train_iter_with_ratio(image, content_weight, style_weight):
  with tf.GradientTape() as tape:
    x = extractor(image)
    loss = compute_loss_with_ratio(x, content_weight, style_weight)

  grad = tape.gradient(loss, image)
  opt.apply_gradients([(grad, image)])
  image.assign(image_clip(image))

start = time.time()
epochs = 1000
save_path = 'img_weight_1e-2.jpg'
for n in range(epochs):
  train_iter_with_ratio(image_prepared, content_weight, style_weight)

display.display(to_image(image_prepared),save_path)
to_image(image_prepared).save(save_path)

end = time.time()
print("Total time: {:.1f}".format(end-start)) 

'img_weight_1e-2.jpg'

Total time: 73.7

# alpha: content, beta: style
# alpha/beta: 1e2
content_image = load_img(content_path)
image_prepared = tf.Variable(content_image)

content_weight=1e2
style_weight=1

@tf.function()
def train_iter_with_ratio(image, content_weight, style_weight):
  with tf.GradientTape() as tape:
    x = extractor(image)
    loss = compute_loss_with_ratio(x, content_weight, style_weight)

  grad = tape.gradient(loss, image)
  opt.apply_gradients([(grad, image)])
  image.assign(image_clip(image))

start = time.time()
epochs = 1000
save_path = 'img_weight_1e+2.jpg'
for n in range(epochs):
  train_iter_with_ratio(image_prepared, content_weight, style_weight)

display.display(to_image(image_prepared),save_path)
to_image(image_prepared).save(save_path)

end = time.time()
print("Total time: {:.1f}".format(end-start)) 

'img_weight_1e+2.jpg'

Total time: 73.0

# alpha: content, beta: style
# alpha/beta: 1e4
content_image = load_img(content_path)
image_prepared = tf.Variable(content_image)

content_weight=1e4
style_weight=1

@tf.function()
def train_iter_with_ratio(image, content_weight, style_weight):
  with tf.GradientTape() as tape:
    x = extractor(image)
    loss = compute_loss_with_ratio(x, content_weight, style_weight)

  grad = tape.gradient(loss, image)
  opt.apply_gradients([(grad, image)])
  image.assign(image_clip(image))

start = time.time()
epochs = 1000
save_path = 'img_weight_1e+4.jpg'
for n in range(epochs):
  train_iter_with_ratio(image_prepared, content_weight, style_weight)

display.display(to_image(image_prepared),save_path)
to_image(image_prepared).save(save_path)

end = time.time()
print("Total time: {:.1f}".format(end-start)) 

'img_weight_1e+4.jpg'

Total time: 72.8

Random Seed

seed = [111,222,333,999]

# seed = 111
random_seed = 111
tf.random.set_seed(random_seed)
opt = tf.optimizers.Adam(learning_rate=0.02, beta_1=0.99, epsilon=1e-1)

def train_iter(image):
  with tf.GradientTape() as tape:
    x = extractor(image)
    loss = compute_loss(x)

  grad = tape.gradient(loss, image)
  opt.apply_gradients([(grad, image)])
  image.assign(image_clip(image))

step = 0
image = tf.Variable(content_image)
start = time.time()
for i in range(1000):
  step += 1
  train_iter(image)

save_path = os.path.join("img_%s_111.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))

end = time.time()
print("Total time: {:.1f}".format(end-start))

'img_1000_111.png'

train step: 1000
Total time: 121.6

# seed = 222

tf.random.set_seed(222)

opt = tf.optimizers.Adam(learning_rate=0.03, beta_1=0.99, epsilon=1e-1)

def train_iter(image):
  with tf.GradientTape() as tape:
    x = extractor(image)
    loss = compute_loss(x)

  grad = tape.gradient(loss, image)
  opt.apply_gradients([(grad, image)])
  image.assign(image_clip(image))
  
image = tf.Variable(content_image)
step = 0
start = time.time()
for i in range(1000):
  step += 1
  train_iter(image)

save_path = os.path.join("img_%s_222.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))

end = time.time()
print("Total time: {:.1f}".format(end-start))

'img_1000_222.png'

train step: 1000
Total time: 122.2

tf.random.set_seed(333)

opt = tf.optimizers.Adam(learning_rate=0.03, beta_1=0.99, epsilon=1e-1)


def train_iter(image):
  with tf.GradientTape() as tape:
    x = extractor(image)
    loss = compute_loss(x)

  grad = tape.gradient(loss, image)
  opt.apply_gradients([(grad, image)])
  image.assign(image_clip(image))

image = tf.Variable(content_image)

step = 0
start = time.time()
for i in range(1000):
  step += 1
  train_iter(image)

save_path = os.path.join("img_%s_333.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))

end = time.time()
print("Total time: {:.1f}".format(end-start))

'img_1000_333.png'

train step: 1000
Total time: 121.6

tf.random.set_seed(999)

opt = tf.optimizers.Adam(learning_rate=0.03, beta_1=0.99, epsilon=1e-1)

def train_iter(image):
  with tf.GradientTape() as tape:
    x = extractor(image)
    loss = compute_loss(x)

  grad = tape.gradient(loss, image)
  opt.apply_gradients([(grad, image)])
  image.assign(image_clip(image))

step = 0
start = time.time()
for i in range(1000):
  step += 1
  train_iter(image)

save_path = os.path.join("img_%s_999.png"%(step))
display.display(to_image(image),save_path)
to_image(image).save(save_path)
print("train step: {}".format(step))

end = time.time()
print("Total time: {:.1f}".format(end-start))

'img_1000_999.png'

train step: 1000
Total time: 121.7

Other Pre-trained Model Weights

def load_img_w2(path):

  img = tf.io.read_file(path)
  img = tf.image.decode_image(img, channels=3)
  img = tf.image.convert_image_dtype(img, tf.float32)

  new_shape = tf.cast([224,224], tf.int32)

  img = tf.image.resize(img, new_shape)
  img = img[tf.newaxis, :]
  return img

content_image_w2 = load_img_w2(content_path)
style_image_w2 = load_img_w2(style_path)

# the layers to extract the content
content_layers = ['block4_conv2'] 

# the layers to extract the style
style_layers = ['block1_conv1',
                'block2_conv1',
                'block3_conv1', 
                'block4_conv1', 
                'block5_conv1']

num_content_layers = len(content_layers)
num_style_layers = len(style_layers)

def vgg_layers_w2(layers):
  # load the pre-trained vgg 
  vgg = tf.keras.applications.VGG19()
  vgg.load_weights("vgg19_weights_tf_dim_ordering_tf_kernels.h5")
  vgg.trainable = False

  output = [vgg.get_layer(name).output for name in layers]

  model = tf.keras.Model([vgg.input], output)
  return model

style_extractor = vgg_layers_w2(style_layers)
style_outputs = style_extractor(style_image_w2*255)
'''
#查看每层输出的统计信息
for name, output in zip(style_layers, style_outputs):
  print(name)
  print("  shape: ", output.numpy().shape)
  print("  min: ", output.numpy().min())
  print("  max: ", output.numpy().max())
  print("  mean: ", output.numpy().mean())
  print()
'''

'\n#查看每层输出的统计信息\nfor name, output in zip(style_layers, style_outputs):\n  print(name)\n  print("  shape: ", output.numpy().shape)\n  print("  min: ", output.numpy().min())\n  print("  max: ", output.numpy().max())\n  print("  mean: ", output.numpy().mean())\n  print()\n'

class NeuralTransfer_w2(tf.keras.models.Model):
  def __init__(self, style_layers, content_layers):
    super(NeuralTransfer_w2, self).__init__()
    self.vgg =  vgg_layers_w2(style_layers + content_layers)
    self.vgg.trainable = False
    self.content_layers = content_layers
    self.style_layers = style_layers
    self.num_style_layers = len(style_layers)

  def call(self, img):
    "Expects float input in [0,1]"
    img = img*255.0
    img_pre = tf.keras.applications.vgg19.preprocess_input(img)
    img1 = self.vgg(img_pre)

    style = img1[:self.num_style_layers]
    style = [gram_matrix(s) for s in style]

    style_dict = {style_name:value
                  for style_name, value
                  in zip(self.style_layers, style)}

    content = img1[self.num_style_layers:]

    content_dict = {content_name:value 
                    for content_name, value 
                    in zip(self.content_layers, content)}


    return {'style':style_dict,'content':content_dict}

extractor = NeuralTransfer_w2(style_layers,content_layers)

content_y = extractor(content_image_w2)['content']
style_y = extractor(style_image_w2)['style']

image_w2 = tf.Variable(content_image_w2)

@tf.function()
def train_iter(image_w2):
  with tf.GradientTape() as tape:
    x = extractor(image_w2)
    loss = compute_loss(x)

  grad = tape.gradient(loss, image_w2)
  opt.apply_gradients([(grad, image_w2)])
  image_w2.assign(image_clip(image_w2))

opt = tf.optimizers.Adam(learning_rate=0.03, beta_1=0.99, epsilon=1e-1)

start = time.time()

epochs = 5
iter_per_epoch = 200

step = 0
for n in range(epochs):
  for m in range(iter_per_epoch):
    step += 1
    train_iter(image_w2)
    print(".", end='')
  display.clear_output(wait=True)
  shape_w2 = tf.cast([274, 365], tf.int32)
  img_w2 = tf.image.resize(image_w2,shape_w2)
  display.display(to_image(img_w2))
  print("Train step: {}".format(step))

save_path = os.path.join("img_%s_w2.png"%(step))
to_image(img_w2).save(save_path)

end = time.time()
print("Total time: {:.1f}".format(end-start))

Train step: 1000
Total time: 23.2

Creative Results

Photo+photo

content_image = load_img(content_path2)
style_image = load_img(content_path)

plt.subplot(1, 2, 1)
show_image(content_image, 'Content Image')

plt.subplot(1, 2, 2)
show_image(style_image, 'Style Image')

style_extractor = vgg_layers(style_layers)
style_outputs = style_extractor(style_image*255)
results = extractor(tf.constant(content_image))
style_results = results['style']
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']

image_prepared = tf.Variable(content_image)

opt = tf.optimizers.Adam(learning_rate=0.02, beta_1=0.99, epsilon=1e-1)
@tf.function()
def train_iter(image):
  with tf.GradientTape() as tape:
    x = extractor(image)
    loss = compute_loss(x)

  style_weight=1
  content_weight=1 
  grad = tape.gradient(loss, image)
  
  opt.apply_gradients([(grad, image)])
  image.assign(image_clip(image))

start = time.time()
epochs = 1000
#image_prepared = basic_style_transfer_prepare(content_path, style_path)
save_path = 'img_c2_c1.jpg'
for n in range(epochs):
  train_iter(image_prepared)

display.display(to_image(image_prepared),save_path)
to_image(image_prepared).save(save_path)

end = time.time()
print("Total time: {:.1f}".format(end-start)) 

'img_c2_c1.jpg'

Total time: 35.4

Drawing + Drawing

content_image = load_img(style_path)
style_image = load_img(style_path2)

plt.subplot(1, 2, 1)
show_image(content_image, 'Content Image')

plt.subplot(1, 2, 2)
show_image(style_image, 'Style Image')

style_extractor = vgg_layers(style_layers)
style_outputs = style_extractor(style_image*255)
results = extractor(tf.constant(content_image))
style_results = results['style']
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']

image_prepared = tf.Variable(content_image)

opt = tf.optimizers.Adam(learning_rate=0.01, epsilon=1e-1)
@tf.function()
def train_iter(image):
  with tf.GradientTape() as tape:
    x = extractor(image)
    loss = compute_loss(x)

  style_weight=1
  content_weight=1 
  grad = tape.gradient(loss, image)
  
  opt.apply_gradients([(grad, image)])
  image.assign(image_clip(image))

start = time.time()
epochs = 1000
#image_prepared = basic_style_transfer_prepare(content_path, style_path)
save_path = 'img_s1_s2.jpg'
for n in range(epochs):
  train_iter(image_prepared)

display.display(to_image(image_prepared),save_path)
to_image(image_prepared).save(save_path)

end = time.time()
print("Total time: {:.1f}".format(end-start)) 

'img_s1_s2.jpg'

Total time: 37.3

style_path3 = 'Claude_Monet_57.jpg'

content_image = load_img(style_path)
style_image = load_img(style_path3)

plt.subplot(1, 2, 1)
show_image(content_image, 'Content Image')

plt.subplot(1, 2, 2)
show_image(style_image, 'Style Image')

style_extractor = vgg_layers(style_layers)
style_outputs = style_extractor(style_image*255)
results = extractor(tf.constant(content_image))
style_results = results['style']
content_y = extractor(content_image)['content']
style_y = extractor(style_image)['style']

image_prepared = tf.Variable(content_image)

opt = tf.optimizers.Adam(learning_rate=0.01, beta_1=0.99, epsilon=1e-1)
@tf.function()
def train_iter(image):
  with tf.GradientTape() as tape:
    x = extractor(image)
    loss = compute_loss(x)

  style_weight=1
  content_weight=10 
  grad = tape.gradient(loss, image)
  
  opt.apply_gradients([(grad, image)])
  image.assign(image_clip(image))

start = time.time()
epochs = 1000
#image_prepared = basic_style_transfer_prepare(content_path, style_path)
save_path = 'img_s1_s3.jpg'
for n in range(epochs):
  train_iter(image_prepared)

display.display(to_image(image_prepared),save_path)
to_image(image_prepared).save(save_path)

end = time.time()
print("Total time: {:.1f}".format(end-start)) 

'img_s1_s3.jpg'

Total time: 37.2